{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/news/fake-news/fake-news-negatives-summarization.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/news/fake-news/fake-news-positives-summarization.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/compressed-fake-news.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/250-news-with-valid-hoax-label-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/600-news-with-valid-hoax-label-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/facts-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/hoax-summaries.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/summarization-augmentation/malaysia-scraping-syazanihussin-summaries.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/hoax.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/facts.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/250%20news%20with%20valid%20hoax%20label.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/indonesian/600%20news%20with%20valid%20hoax%20label.csv\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/news/fake-news/malaysia-scraping-syazanihussin.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string)\n",
    "    string = string.replace('SPPPPLIIIT>', ' ').replace('SPPPPLIIIT', ' ')\n",
    "    string = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('250 news with valid hoax label.csv', sep = ';',encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = [], []\n",
    "\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,0])\n",
    "    Y.append(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('600 news with valid hoax label.csv', sep = ';',encoding = \"ISO-8859-1\")\n",
    "\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,0])\n",
    "    Y.append(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('250-news-with-valid-hoax-label-summaries.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('600-news-with-valid-hoax-label-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('facts.csv', sep = ',',encoding = \"ISO-8859-1\")\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,1])\n",
    "    Y.append(df.iloc[i,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('hoax.csv', sep = ',',encoding = \"ISO-8859-1\")\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,1])\n",
    "    Y.append(df.iloc[i,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('hoax-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('facts-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('malaysia-scraping-syazanihussin.csv')\n",
    "for i in range(len(df)):\n",
    "    X.append(df.iloc[i,0])\n",
    "    Y.append(df.iloc[i,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('malaysia-scraping-syazanihussin-summaries.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X.extend(i[0])\n",
    "    Y.extend([i[1]] * len(i[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !unzip compressed-fake-news.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "\n",
    "X_, Y_ = [], []\n",
    "\n",
    "for file in glob('positive/*.json'):\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    X_.extend(data)\n",
    "    Y_.extend(['fake'] * len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in glob('negative/*.json'):\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    X_.extend(data)\n",
    "    Y_.extend(['real'] * len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('fake-news-negatives-summarization.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X_.extend(i)\n",
    "    Y_.extend(['real'] * len(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('fake-news-positives-summarization.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "for i in data:\n",
    "    X_.extend(i)\n",
    "    Y_.extend(['fake'] * len(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 59935/59935 [00:21<00:00, 2777.64it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(50284, 50284)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "selected_X, selected_Y = [], []\n",
    "mapping_label = {'fake': 0, 'hoax': 0, 'real': 1, 'valid': 1, 'fact': 1}\n",
    "\n",
    "for i in tqdm(range(len(X_))):\n",
    "    try:\n",
    "        t = cleaning(X_[i])\n",
    "        if len(t) > 100:\n",
    "            selected_X.append(t)\n",
    "            selected_Y.append(mapping_label[Y_[i].lower()])\n",
    "    except:\n",
    "        pass\n",
    "        \n",
    "len(selected_X), len(selected_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45255, 5029)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(selected_X, selected_Y, test_size = 0.1)\n",
    "len(train_X), len(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = train_X + selected_X\n",
    "train_Y = train_Y + selected_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "train_X, train_Y = shuffle(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from malaya.text.bpe import WordPieceTokenizer\n",
    "\n",
    "tokenizer = WordPieceTokenizer('BERT.wordpiece', do_lower_case = False)\n",
    "# tokenizer.tokenize('halo nama sayacomel')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tokens(X, Y, maxlen = 1024):\n",
    "\n",
    "    input_ids, input_masks = [], []\n",
    "    actual_l = []\n",
    "\n",
    "    for i in tqdm(range(len(X))):\n",
    "        text = X[i]\n",
    "        tokens_a = tokenizer.tokenize(text)\n",
    "        tokens = [\"[CLS]\"] + tokens_a + [\"[SEP]\"]\n",
    "        input_id = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(input_id)\n",
    "\n",
    "        if len(input_id) <= maxlen:\n",
    "            input_ids.append(input_id)\n",
    "            input_masks.append(input_mask)\n",
    "            actual_l.append(Y[i])\n",
    "            \n",
    "    return input_ids, actual_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 95539/95539 [15:20<00:00, 103.74it/s]\n"
     ]
    }
   ],
   "source": [
    "train_input_ids, train_actual_l = get_tokens(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5029/5029 [00:47<00:00, 104.84it/s]\n"
     ]
    }
   ],
   "source": [
    "test_input_ids, test_actual_l = get_tokens(test_X, test_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('relevancy-fastformer.pkl', 'wb') as fopen:\n",
    "    pickle.dump([train_input_ids, train_actual_l, test_input_ids, test_actual_l], fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
